{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Ch `10`: Concept `02`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Import the relevant libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define the RNN model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class SeriesPredictor:\n",
    "\n",
    "    def __init__(self, input_dim, seq_size, hidden_dim=10):\n",
    "        # Hyperparameters\n",
    "        self.input_dim = input_dim\n",
    "        self.seq_size = seq_size\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        # Weight variables and input placeholders\n",
    "        self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')\n",
    "        self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')\n",
    "        self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])\n",
    "        self.y = tf.placeholder(tf.float32, [None, seq_size])\n",
    "\n",
    "        # Cost optimizer\n",
    "        self.cost = tf.reduce_mean(tf.square(self.model() - self.y))\n",
    "        self.train_op = tf.train.AdamOptimizer().minimize(self.cost)\n",
    "\n",
    "        # Auxiliary ops\n",
    "        self.saver = tf.train.Saver()\n",
    "\n",
    "    def model(self):\n",
    "        \"\"\"\n",
    "        :param x: inputs of size [T, batch_size, input_size]\n",
    "        :param W: matrix of fully-connected output layer weights\n",
    "        :param b: vector of fully-connected output layer biases\n",
    "        \"\"\"\n",
    "        cell = rnn.BasicLSTMCell(self.hidden_dim)\n",
    "        outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)\n",
    "        num_examples = tf.shape(self.x)[0]\n",
    "        W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])\n",
    "        out = tf.matmul(outputs, W_repeated) + self.b_out\n",
    "        out = tf.squeeze(out)\n",
    "        return out\n",
    "\n",
    "    def train(self, train_x, train_y):\n",
    "        with tf.Session() as sess:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for i in range(1000):\n",
    "                _, mse = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})\n",
    "                if i % 100 == 0:\n",
    "                    print(i, mse)\n",
    "            save_path = self.saver.save(sess, 'model.ckpt')\n",
    "            print('Model saved to {}'.format(save_path))\n",
    "\n",
    "    def test(self, test_x):\n",
    "        with tf.Session() as sess:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            self.saver.restore(sess, './model.ckpt')\n",
    "            output = sess.run(self.model(), feed_dict={self.x: test_x})\n",
    "            return output\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now, we'll train a series predictor. Let's say we have a sequence of numbers `[a, b, c, d]` that we want to transform into `[a, a+b, b+c, c+d]`. We'll give the RNN a couple examples in the training data. Let's see how well it learns this intended transformation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 92.1852\n",
      "100 61.1175\n",
      "200 27.0341\n",
      "300 13.9523\n",
      "400 9.39037\n",
      "500 7.08643\n",
      "600 5.50997\n",
      "700 4.12571\n",
      "800 3.12016\n",
      "900 2.42311\n",
      "Model saved to model.ckpt\n",
      "\n",
      "Lets run some tests!\n",
      "\n",
      "When the input is [[1], [2], [3], [4]]\n",
      "The ground truth output should be [[1], [3], [5], [7]]\n",
      "And the model thinks it is [ 0.96018004  2.76944828  5.35826826  7.3706851 ]\n",
      "\n",
      "When the input is [[4], [5], [6], [7]]\n",
      "The ground truth output should be [[4], [9], [11], [13]]\n",
      "And the model thinks it is [  4.17302942   9.161376    11.13204765  11.64120388]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    predictor = SeriesPredictor(input_dim=1, seq_size=4, hidden_dim=10)\n",
    "    train_x = [[[1], [2], [5], [6]],\n",
    "               [[5], [7], [7], [8]],\n",
    "               [[3], [4], [5], [7]]]\n",
    "    train_y = [[1, 3, 7, 11],\n",
    "               [5, 12, 14, 15],\n",
    "               [3, 7, 9, 12]]\n",
    "    predictor.train(train_x, train_y)\n",
    "\n",
    "    test_x = [[[1], [2], [3], [4]],  # 1, 3, 5, 7\n",
    "              [[4], [5], [6], [7]]]  # 4, 9, 11, 13\n",
    "    actual_y = [[[1], [3], [5], [7]],\n",
    "                [[4], [9], [11], [13]]]\n",
    "    pred_y = predictor.test(test_x)\n",
    "    \n",
    "    print(\"\\nLets run some tests!\\n\")\n",
    "    \n",
    "    for i, x in enumerate(test_x):\n",
    "        print(\"When the input is {}\".format(x))\n",
    "        print(\"The ground truth output should be {}\".format(actual_y[i]))\n",
    "        print(\"And the model thinks it is {}\\n\".format(pred_y[i]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
